import requests
import csv

class Crawler:
    def __init__(self,headers):
        self.headers = headers



    def donwload_content(self,url):
        """
        用来下载网页，返回网页内容
        :param url: 网页地址
        :return: 网页内容
        """
        response = requests.get(url, headers=self.headers).text
        return response


    def save_to_file(self,filename, content):
        """
        保存到文件中
        :param filename: 文件名
        :param content: 内容
        :return:
        """
        with open(filename, mode="a", encoding="utf-8") as f:
            f.write(content)


    def main(self):
        url = "https://gitee.com/organizations/openeuler/projects?page="
        # result = donwload_content(url)
        # save_to_file("result.html", result)
        for i in range(1, 20):
            result = self.donwload_content(url+str(i))
            self.save_to_file("result.html", result)
            print("success crawling page "+str(i))


    def repo(self):
        url = "https://gitee.com"
        with open("OpenEuler_repo.csv", mode="r", encoding="utf-8") as f:
            reader = csv.reader(f)
            header = next(reader)
            for row in reader:
                title, href = row[0], row[1]
                print("reading "+url+str(href))
                result = self.donwload_content(url+str(href))
                self.save_to_file("repo_result.html", result)


